{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from sklearn import datasets\n",
    "\n",
    "digits = datasets.load_digits()\n",
    "X = digits.data\n",
    "y = digits.target.copy()\n",
    "\n",
    "y[digits.target==9] = 1\n",
    "y[digits.target!=9] = 0\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=666)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\yinahe\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.9755555555555555"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "log_reg = LogisticRegression()\n",
    "log_reg.fit(X_train, y_train)\n",
    "log_reg.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "decision_scores = log_reg.decision_function(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3deZxcVZ338c+vtq7eO0l3ts4ekkDYoQn7IqCGUYnLqEFBVEaEGXzw0RmFYcaXOj4zLjP66AyOT1RUUEAUwYyDg2wqIkkIhEBCyE6SztpJuju9Vtdynj9udeh0Okl1d3XfWr7v16tfdevWTdXvUsmX0+eee4455xARkfwX8LsAERHJDgW6iEiBUKCLiBQIBbqISIFQoIuIFIiQXx9cW1vrZsyY4dfHi4jkpRdffHG/c65uoNd8C/QZM2awcuVKvz5eRCQvmdm2Y72mLhcRkQKhQBcRKRAKdBGRAqFAFxEpEAp0EZECccJAN7N7zGyfma05xutmZt8xs01m9oqZnZP9MkVE5EQyaaH/GFh4nNevAeakf24G/nP4ZYmIyGCdcBy6c+6PZjbjOIcsAu513jy8y8ysxswmOed2Z6nGI217HjY/PSJvLZLXSirg/FshFPG7El9t2NvGb1bv8ruM47rqlAmcObUm6++bjRuL6oEdfZ43pvcdFehmdjNeK55p06YN7dMaV8AfvzG0PytSsNLrGkw+B2Ze6m8pPlvyxy388sVGzPyu5NjGV0VzNtAH+s824KoZzrklwBKAhoaGoa2scfHt3o+IvGnPq/C9S6Cr2e9KfNcVTzK7rpynPnuF36WMumyMcmkEpvZ5PgXI7d93RApNtNp77G71t44cEIunKAkF/S7DF9kI9KXAR9KjXS4AWkes/1xEBhZN//redRCSCSjipSVjiSQl4eIckZ3JsMUHgOeBeWbWaGY3mdktZnZL+pDHgC3AJuD7wF+PWLUiMrBIBQTC8MQX4J/GwQPX+V2Rb2KJFCWh4gz0TEa5HPdvRnp0y99krSIRGbxAAN73A9i/EV7/Dex5xe+Ksso5x9b9HXTFk5RFQsysLT/msbFEiprS8ChWlzt8mz5XRLLs1Hd7j13N8NJP/K0ly1Zua+b933v+8HMzqC4N8/mFJ9MwfQxzJlQefi0WT1JSWeJHmb5ToIsUmmg19LRDMg7Bwmip7mntBuBL155KPJmipTPO42v3cOevXgXgqpPHc+sVs5lZW053PElJuDgviirQRQpNafoC6ZNfhNAItFRPfS9MPC3773scnT0JAN46fwKTa0oBuO3Kk3ilsZUHVmznkVU7eer1fYePH18ZHdX6coUCXaTQTDwdwuWw/HvZf+9UAlp2wPu+n/33Po72WBKA8pI3IysaDrJg5lgWzBzL5xbO46l1+9jV0sV3f7+Z06dUj2p9uUKBLlJopl8Ed43QrSBLrhiRm5fW7Gzl9gdX0ZNMHbE/Egxw94fPoTPmtdDLIgN3pUyqLuX6C6YD8MnLZx/zuEKnQBeRzEWrR+TmpVU7Wtjc1ME7zphESfDNIYe/eWU3tz/wMjVlYSKhAOHgiYcjVhfpCBdQoIvIYERroHUnJGJH7g+EIDD0VnFbdxyAf3v/mUT7XNCcN7GSf/nt6wDUp/vO5dgU6CKSubKxcGAjfGX8kftLx8LtqyFaNaS3betOEA7aUTcEffLy2SxeMI11uw8xdWzZUKsuGgp0EcncRZ+C6qkcMf/e3tdgzS+hbfeQAn3r/g427m2nMhrGBpgisbo0zAWzxg2j6OKhQBeRzI2dBZd+5sh9G5/0An2Ifevv/M6zdPQkmT5OLfDhKs4JD0Qke3pneuxqGdIf7+jxhiT+wzvmZ6uioqVAF5Hh6b2RaeUPh/THK0pC3HTJTN46f0IWiypOCnQRGZ6qyd7jhv+BVOr4x/aTSKZojyWoihbvUMNsUqCLyPBEyuFt/8fb7mkb1B9dtcPrpqkq1eW8bFCgi8jwDXHFpDU7vePPmJL99TWLkQJdRIavtx+9bS/0dHgzPWagtcs77qwRWDC5GOn3HBEZvtKx3uMPr/YeI5XejUblxx8/3tIZp7IkRDAw0FrzMlgKdBEZvqnnwzu+6c3D3rQBXv4ptG4/YaAf6opTXaYLotmiQBeR4QuG4LybvO03nvMCPYNx6S1d8aKeTCvb1IcuItnV25+ewQXS1q44NWqhZ41a6CKSXb0jXl76CTS+cPTrs98CJ3l97S2dPcybWHn0MTIkCnQRya7y8VB3MuxY4f30Fe+CbX/m9YoF7GzuYn97DwtKI/7UWYAU6CKSXaEI/M3ygV/7+fW4A5t5z91/pivuzeEyubo41/8cCQp0ERk9oSipni664kn+6pKZvPvsenW5ZJEuiorIqHDOsXJnJ+0d7QCcMqmK0+qrM1pWTjKj/5IiMio6epKs3ddDsqcLgDHlGt2SbQp0ERkVzR09xIhQgne7/9QxWtAi29SHLiKjYsfBTmKEKQ0keP7OK5lUrUWfs00tdBEZcfsOdfOhHywn5sIEXJJJFepuGQlqoYvIiNt2sBOAeVPqYC/w289BsE+oz18E0y/yp7gCokAXkRF3oD0GwOkNl8AzD3mLSveKtcPBLQr0LFCgi8iI29/eA0B07lVw3uYjX7x30ZAXmJYjZdSHbmYLzWy9mW0yszsGeH2amT1jZqvM7BUz+4vslyoi+epghxfoY8sHuM0/Wj3olY5kYCdsoZtZELgbeCvQCLxgZkudc6/1OewfgIecc/9pZvOBx4AZI1CviOShA+0xqqIhIqEB2pDRauhqhs6DR78WDEOJ7iTNVCZdLguATc65LQBm9iCwCOgb6A6oSm9XA7uyWaSI5K9dLV1sampnXEXJwAeU1ULHPvj6zKNfswB8/HGYumBkiywQmQR6PbCjz/NG4Px+x3wR+J2ZfQooB64e6I3M7GbgZoBp06YNtlYRyUPXfX8Z2w50cslJtQMfcP4tUDUZXOrI/e374Nl/hdYdCvQMZRLoAy325/o9vw74sXPu38zsQuA+MzvNuSO/IefcEmAJQENDQ//3EJECk0o5Gpu7eO/Z9dz1jlMGPqhyAiz4xNH7D2z2Aj3DBacls4uijcDUPs+ncHSXyk3AQwDOueeBKHCM/x2LSLFo7uwhmXKcMaX62F0uxxJMX0BVoGcsk0B/AZhjZjPNLAIsBpb2O2Y7cBWAmZ2CF+hN2SxURPJPU3r8eV3lEOY8773xKNmTxYoK2wkD3TmXAG4DHgfW4Y1mWWtmXzaza9OHfRb4hJmtBh4APuqcU5eKSJHb3+aFcV3lIFvn8GYLPZXIYkWFLaMbi5xzj+ENRey77wt9tl8DLs5uaSKS75rau4EhBnogHU9qoWdMk3OJyIhpavO6XGorhrBuqPrQB02BLiIjpqktRjQcoKJkCLOMHO5DV6BnSoEuIiOmqS1GXWUJZgONfj6BQNC7sSilQM+UAl1ERsQNP1zOr1fvom6wwxX7CoTVhz4Imm1RRLIulXL8efMBzp02hs++bd7Q3ygYgaRGuWRKLXQRybr9HTGSKce1Z03mwtnjhv5GwZBa6IOgQBeRrNt3yBvdMn4oNxT1FQirD30QFOgiklW7W7u4f8V2ACZUDaP/HCBcCj2dWaiqOKgPXUSy6o6HX+UPG7yZPyZWD7OFHq2G2KEsVFUc1EIXkazakV4QujQcpHY4I1xAqxkNkgJdRLJm5RsH2bK/g8vn1vH8nVcSDg4zYkproPmNrNRWDBToIpI1T7y2F4CPXTyDmrIh3O7fX9k4aNsNL907/PcqAgp0EcmaQ90JaitKuGLe+Oy84WWf8x4PbM7O+xU4BbqIZE1bd5zKaBbHWlTXQ/l46G7J3nsWMAW6iGRNW3ciu4EOujA6CBq2KCJZk/UWOniBvmMF/Po273lJJVz5jxApy+7nFAC10EUka5raY4wrH+ZQxf7mvBWcg01PwfrHYNl3YeeL2f2MAqFAF5GsSCRT7G7pZurY0uy+8RV3wGfXeT83POLtU5/6gNTlIiLDFkskeX13G4mUY8qYEewKidZ4j+pTH5ACXUSG7a5H1vDLFxsBqK/Jcgu9r9J0oDdvg44DUD6MmRwLkLpcRGTY/vuV3Ye3p48bwRZ6pBKCJfDHr8M3ZsO+dSP3WXlIgS4iwzarrhyAH3ykgenjykfugwIBrx/90s8CDg7tGrnPykMKdBEZtsbmLm64YDpXz58w8h8242I45VpvOxEb+c/LIwp0ERmW5o4eWrviTBs7iuPCQ+lpeRPdo/eZeUCBLiLDcs23nwXg7Gk1o/ehofRYdwX6ERToIjJksUSSPYe6mTa2jHOnjxm9D1YLfUAKdBEZspZOb73Pmy+bhZmN3gcfbqGrD70vBbqIDIlz7vDqRGOyMff5YITTY93VQj+CbiwSkSG569E13L/cWwx6XMUoB3ow3UKPK9D7UqCLyJBs2NPGrLpybrl8Ng2j2X8O3nj0YAQSXaP7uTlOgS4iQ3Kws4dTJlXxgYap/hRQUgXdh/z57ByVUR+6mS00s/VmtsnM7jjGMR8ws9fMbK2Z3Z/dMkUk1zR39DB2tPvO+yqt0SRd/ZywhW5mQeBu4K1AI/CCmS11zr3W55g5wJ3Axc65ZjPL0oKCIpKLuuNJWrri1FZkee7zwYhWaxrdfjJpoS8ANjnntjjneoAHgUX9jvkEcLdzrhnAObcvu2WKSK544rW93Hb/KpyDk8ZX+FdItAZ2r4Znv+lfDTkmk0CvB3b0ed6Y3tfXXGCumT1nZsvMbOFAb2RmN5vZSjNb2dTUNLSKRcRX3392C3/a1MT8SVWcN2OUL4b2Nfft4FLw1JcglfSvjhySSaAPdLeA6/c8BMwBrgCuA35gZkfdB+ycW+Kca3DONdTV1Q22VhHxkXOOR1ftZNX2Zq46eQKP3X4p46ui/hV0/ifhss952zFdHIXMAr0R6HsZewrQf87KRuDXzrm4c24rsB4v4EWkQKzZeYhP//xl4kk3+uPOjyVa7T12qS8dMgv0F4A5ZjbTzCLAYmBpv2MeBd4CYGa1eF0wW7JZqIj4a93uN1vBU8aM4KpEg1GqJen6OuEoF+dcwsxuAx4HgsA9zrm1ZvZlYKVzbmn6tbeZ2WtAEvg759yBkSxcREZXY7N3m//Dt17I6fWjOLPi8fS20DXaBcjwxiLn3GPAY/32faHPtgM+k/4RkQLU1B6jtqKEc6eP9buUN2nR6CNoci4ROSHnHA+s2EFtrvSd9+rtclEfOqBAF5EMbE/Pqji7zsdx5wM53OWiFjoo0EUkA2t2ehdEb7l8ts+V9BNJ/w/m6a9AMuFvLTlAgS4iJ/R/n9xAwGDuxBxroZtBzXRIxuDgZr+r8Z0CXUSOa9+hbjbua+e0+mpKQkG/yznaO9K3/qvbRYEuIsf3wApv5o/Pvf1knys5BvWjH6ZAF5Hj2nOom2g4wCVzav0uZWC9I13a9/pbRw5QoIvIMe091M0DK7YzqzbH+s77KhvnPb74Y1/LyAUKdBE5pt+v92bCvmj2OJ8rOY6ysVAxEUxxpv8CInJM63a3AfD5a3K0/7zX1AXqQ0eBLiLHcN/zb/DjP7/BxKoo4WCOR0W0WneLokAXkWN4+nWvu+XfP3S2z5VkoLQGOprg5zfA2kf8rsY3CnQROUoy5Vi5rZnrFkzjvBk5NBnXscx6C4w/BTY9BcuX+F2NbxToInKEprYYdz+zibbuhL9LzA3GSVfBrc/B7LcUdV96RtPnikjx+I+nN/KT57dREgpw0ewcHXt+LNEaBbqISK8n1+3j4pPG8f2PNFAWybOIiFbDoUa/q/CNulxE5LDfvLKLnS1dXDS7Nv/CHCBS5j1ufdbfOnyiQBeRw55e541sufGiGf4WMlTzF3mPrcXZSlegiwgAqZTjjxv3864zJ1NRkoetc4Cqeu+xSPvRFegiAsC6PYfY3x7j8rl1fpcydCVV3mORLhqtQBcRAP6woQmAy3J1VsVMBEMQLoNXHvK7El8o0EUEgD+sb2L+pCrGV0X9LmV4QiXQddDvKnyhQBcR2rrjvLitmcvn5XF3S69zboRYOzjndyWjToEuIizbcpBEyuV3/3mv0hpIxSHe5Xcloy5PL2WLSDat3dWKGZw1tcbvUoavd0m67c9DeS2EolA711tQusAp0EWEzU0dTBlTSjScg4tAD1bFRO/xp+99c9+HHoK5b/ennlGkQBcRXtrWzJwJObzM3GDMeRtc/ytIdHvj0R+9tWhuNFIfukiRW77lADtbuphdVyCBHgx5sy+e/A449T3eviIZl65AFylyj768E4CbL5vlcyUjIFwKwZKiuXNUgS5SxP60cT8PrNjBorMmMyHfx58fS0klrP6531WMCgW6SJFyznHXo68CsPi8aT5XM4LCZdDV7HcVoyKjQDezhWa23sw2mdkdxznuL83MmVlD9koUkWx7at1ezv3Kk2w70MnX//IMLpw9zu+SRs7Z10MyBsm435WMuBMGupkFgbuBa4D5wHVmNn+A4yqB/wUsz3aRIpJd9zy3lVDA+OsrZnPtmZP9LmdklabH1hdBP3omLfQFwCbn3BbnXA/wILBogOP+Cfg60J3F+kQki+LJFP/46Bqe23SAD50/jc8tPLkwxp4fT++NRgp0AOqBHX2eN6b3HWZmZwNTnXO/yWJtIpJlL7xxkPuWbQPg/Q1Tfa5mlETTLfSuwh+6mMmNRQPdL3t41hszCwDfAj56wjcyuxm4GWDatAK+CCOSo77229cBeOGuq6mrLPG5mlFyuIVe+IGeSQu9Eej7v/IpwK4+zyuB04Dfm9kbwAXA0oEujDrnljjnGpxzDXV1BTAJkEge2Xuom9WNrZwxpbp4whzUh97PC8AcM5tpZhFgMbC090XnXKtzrtY5N8M5NwNYBlzrnFs5IhWLyKD9z5rdnP/PTwHwmbfO9bmaUaYW+puccwngNuBxYB3wkHNurZl92cyuHekCRWT4ntt0AIBvLz6LK+aN97maUdbbh/78d+GhGwu6pZ7R5FzOuceAx/rt+8Ixjr1i+GWJSDZt2NvGGVOqWXRW/YkPLjThUjjzOti1Cl57FBo+DrMu97uqEaE7RUUK3MGOHpZvPcil+bxW6HCYwXu+B+/7ofe8gLteFOgiBa53ZEvRdbX0VwTj0RXoIgWspbOHn6/cQU1ZmHOnjfG7HH/1jnbZ+ZK/dYwgBbpIgdp7qJv3f+95AD6/8GQCgcJfgu24Iun53l/8UcFO1qVAFylQ//70Rjbua6dh+hg+UCx3hR6PGVyenluwvcnfWkaIlqATKUCPrtrJT5dt523zJ7DkI5r89LAp53mPBXphVC10kQLTHU/y+YdfobIkxF+/5SS/y8ktBX5hVIEuUmA+cs8KYokUf7dwHmdNrfG7nNzSe2H0yS/BG8/5W8sIUKCLFJC1u1pZsfUgi8+bynULNAHeUWqmw8nvhKZ1sG7piY/PMwp0kQLRHU9y049XEgkF+NRVcwgH9c/7KKEILP4ZVE0uyOl09Y2LFIg/bGhiz6Fu/uU9p1NfU+p3ObktWlOQ/egKdJECEEsk+eR9L1JfU8q7Cn1JuWyIVhfkSBcFukgB+N3avQC888xJREL6Z31CpWqhi0iO+q/Vu4iGA/zt2+b5XUp+iFYr0EUk96za3syT6/by8Ytn6kJopqI1uigqIrll6/4OPrhkGTVlET560Qy/y8kf0RqId0Ay7nclWaVAF8ljH/3RCnoSKe7/xPmMr4r6XU7+6L1j9CfvgnvfDXvW+FtPlijQRfLUxr1tbDvQyTWnTeTkiVV+l5NfZl0Os67wtrc8A5uf8rOarNHkXCJ56NFVO/nHR9cwrjzCF6891e9y8k/dPPjIr8E5+PK4grlAqha6SJ757au7+fTPX6YkHOBHHzuPCepqGTqzghrxoha6SJ5IJFP8+M9v8O0nN3J6fTW/uOVCouGg32Xlv2g1HNjk9aNPPM3vaoZFLXSRPHH/iu185b/X0RVP8sVrT1WYZ0tVPWz5PXzvYmjZ7nc1w6JAF8kT9z6/jdPrq9nwlWs4d3qRrw+aTe//EVz9JW+7ba+/tQyTAl0kD3T2JNjc1M6VJ4/X2qDZVjEepl/kbed5X7oCXSQPrN7RinNwWn2136UUpmh64Ys8n7BLgS6S47Y0tXP7g6sYUxamQV0tI6P3RqNn/hme/oq/tQyDAl0kh63d1crHfvwCPckUD958IWPKI36XVJjK6+CsD0O8C1be43c1Q6ZAF8lBiWSKO3/1Ku/4zp/YdqCTL7xzPvMmVvpdVuEKBODd34UzF3v96M75XdGQaBy6SA5Jphw/em4rv1jZyPq9bVw+t44vXnsqM2vL/S6tOESrIZWAng4oqfC7mkFToIvkCOcc7/vPP/PyjhZqysJ88wNn8t5zpvhdVnEpTV8c3bEcTrrK31qGQIEukiN+tnw7L+9o4ZOXzeKOa07GTMMTR11Vvff40/fCXXshnF/TKqgPXcRna3a2ct2SZfzDo2uorynlU1fNUZj7ZfZVcM6N3nZXs7+1DEFGgW5mC81svZltMrM7Bnj9M2b2mpm9YmZPmdn07JcqUlicc3zj8dd557//idWNLbz3nHqe+MxlVJToF2ffBAJvTqubh2PST/g3x8yCwN3AW4FG4AUzW+qce63PYauABudcp5ndCnwd+OBIFCxSCPa0dvPJ+1ayurGVq08Zz9fedwbjKkr8LkvgzTHpeXjXaCYt9AXAJufcFudcD/AgsKjvAc65Z5xznemnywBdyRE5ht+v38fCb/+R1Y2tLDx1It+7/lyFeS7pvTB6/weg+5C/tQxSJr/b1QM7+jxvBM4/zvE3Ab8d6AUzuxm4GWDatGkZlihSGJxzfOuJDdz9+81MG1vGkhsaWDBzrN9lSX91p0D5eOjYBwc2Qv25fleUsUxa6ANdnRlw1L2ZXQ80AN8Y6HXn3BLnXINzrqGuri7zKkXynHOOzz/8Ct95ehOLzprMf33qEoV5roqUwfu+720nYv7WMkiZtNAbgal9nk8BdvU/yMyuBu4CLnfO5dd/BZEsc86xry1GS2ecl7Y3c//y7by6s5VPXDqTv/+LUzSKJdeF0sMV413+1jFImQT6C8AcM5sJ7AQWAx/qe4CZnQ38P2Chc25f1qsUyQPOOTbsbefhlxr571d2s7PlzTA4aXwFt181h9s1JDE/hNLXNAqthe6cS5jZbcDjQBC4xzm31sy+DKx0zi3F62KpAH6R/su63Tl37QjWLZITunqSfOvJDazZ2cquli7eOOCNDTh3+hhuuHA6U8eUURoJcNmcOkJB3faRN3pb6Iluf+sYpIwGvDrnHgMe67fvC322r85yXSI5raWzh7uf2cR9y7bRHU9x1tQapo4t40PnT+PSOXWcMqnK7xJlOA4HeoG10EXEc6g7zoY9baza3sI3freeeDLFFXPr+KtLZ3HxSbV+lyfZdDjQC68PXaQoxZMpntu0n/9avZtlWw4c0SfeMH0MX7z2VK0gVKgKtQ9dpJikUo4n1+3lp8u3s2LrAbrjKSqjIS6eXcuHL5jGrNoKpo8rY+6ESoJa27Nw9bbQ1/wKLrjV31oGQYEugjdC5QfPbuWbT2ygK55k2tgyFp83jYtPquWyubWUhIJ+lyijKVzqParLRSQ/NLXF2Li3jRfeaObXL+9ky/4O3jKvjnedOZl3nTmZsEalFC8zOOODsP15vysZFAW6FIWeRIqdLV088lIjf9i4nwPtMRqb32x9XTR7HLdcMZv3nztF48TFE63Ouwm6FOhSEJxzbNzXzp7Wbl5pbGHr/k5e33OI1q44bd0JWrvigNfwOm/6WM6eNobrL5jOGfXVTBtXxpQxZT6fgeScaI03Odf633LEDCiRcphxifeXKcco0CWnJZIpDnb2sLulm1giRVNbjFfTN/HEkyniyRR7DnWzcW87sUTq8J+bWBVl6thS5k2spDwSYnxlCTXlEa48eTz1NaU+npHkjeopgIMHFh/92k1PwNQFo17SiSjQZdS1dsVpbO5k3e429h7qpq07wf72GE1tMQ51x72Qbo1xsCNGaoBp4MxgyphSSsNBwsEAY8sj3HDBdGaPr2BmbTmnTKyiuiw8+icmheXs62Hy2d6i0b0OboGHb4K2Pf7VdRwKdMma7niS1q44zZ09tHcn2Lq/g31tMbYf6GTr/g52tnTRHnuz+6NXOGiMKy+hrrKE6tIwkVCAWbUVTBtbRiQUIBoOMLO2grJIkOrSMCeNryAa1qgTGWGBIEw648h95ekbyHK0b12BXuS81nA3+9tjJFOORMrRFU96/c6dPRzo6Dm8P5lyHOqK05NMkUimn3fH6Ygl2NXSzZ5DA897UVsRYfq4cs6fNZaKkhB1FSXMqqtg9vhyZowrVzhL/ji8mlFuLk+nQM8xiWTqcKB2xBK0xRK0dyfo7EnSHU/SFkvQ3ZOkJ5kiFk/S2ZOkM54knkiRSLnD/cqxRIrOWJJYMkVPIkVPIunt60kST6a8kE46epKpE9YUDJj3Y0Z5SYiSUIBw0NtXEQ1TWRLiwtnjmF1XTk1ZhDFlEcpLgkwfV86k6qgCWwpHpBIwtdALQTyZoiOWIOUg5RyplCPlIHl42x1+LRZP0dQeozOWoD2WoCueJBZPEUskOdSdYFdLF61dcbrjSdpjSZraumntihNPDrh2yIDMoDQcpCwSIhI0QsEAoaARCQaIhAKUR0LURMKEgwFKQt6+sojX7+wFsvd8THmEKTWlBANGKGiUhIJUl4aojIapqyghoDsiRTyBgNdK71ILPSue27Sf363dQ3c8dThIk71Bmu4WSDqvpdodT5JKeQGb7BvAh8PXEUukiCdSh9/DOZd+/ejQ7kmcuDWbiUgowOTqKGPKI5SGg9TXhDlrajU1ZRHKwkGi4SAV0RAVJemfaIiySJDScJCKkhClkSAloSDhoGnMtMhoy+Hx6XkX6Ov3tPHIqp2URoIEzQj06Q4I9HkMB41oKEggAOFAgIAZAfOODRiHtyMhr/UaMC8cA0b6GMOMw+/X2xqujIYJpo+xfu8XMCMQ8LZDgQATqkooT4dyacQL6pJQgFBAQSySt0pr1IeeLR+/ZCYfv2Sm32WISLGKVsOmp7yZGHtnZcwRmqxCRFQ8AuYAAAZZSURBVGQwymrBJeHVX/hdyVEU6CIig/HOb3mP7Xv9rWMACnQRkcEorYFgSU5eGFWgi4gMVmlNTg5dVKCLiAxWtBpe+gm07vS7kiMo0EVEBmv2ld7j7pf9raMfBbqIyGCdf4v3mGP96Ap0EZHB6p2kK8f60fPuxiIREd/1BnrjClg7OfM/N3YmTDpzZGpCgS4iMniBIFRNgbWPeD+ZKqmCO3eMWFkKdBGRobjl2cGtXLTqPlj2XYh3Qzg6IiUp0EVEhqJsrPeTqXEneY/dLRCeOCIl6aKoiMhoGIULqQp0EZHRUFrjPd7/AVjz8Ih8hLpcRERGw5Tz4KwPQ087RGtG5CMyCnQzWwh8GwgCP3DOfbXf6yXAvcC5wAHgg865N7JbqohIHotWw7u/O6IfccIuFzMLAncD1wDzgevMbH6/w24Cmp1zJwHfAr6W7UJFROT4MulDXwBscs5tcc71AA8Ci/odswj4SXr7l8BVpjXWRERGVSaBXg/0HQnfmN434DHOuQTQCozr/0ZmdrOZrTSzlU1NTUOrWEREBpRJoA/U0nZDOAbn3BLnXINzrqGuri6T+kREJEOZBHojMLXP8ynArmMdY2YhoBo4mI0CRUQkM5kE+gvAHDObaWYRYDGwtN8xS4Eb09t/CTztnDuqhS4iIiPnhMMWnXMJM7sNeBxv2OI9zrm1ZvZlYKVzbinwQ+A+M9uE1zJfPJJFi4jI0TIah+6cewx4rN++L/TZ7gben93SRERkMMyvnhEzawK2+fLhnlpgv4+fPxqK4RxB51lIiuEcYXjnOd05N+CoEt8C3W9mttI51+B3HSOpGM4RdJ6FpBjOEUbuPDU5l4hIgVCgi4gUiGIO9CV+FzAKiuEcQedZSIrhHGGEzrNo+9BFRApNMbfQRUQKigJdRKRAFFWgm9lZZrbMzF5Oz/q4IL3fzOw7ZrbJzF4xs3P8rnW4zOxTZrbezNaa2df77L8zfZ7rzeztftaYLWb2t2bmzKw2/bxgvk8z+4aZvZ4+j0fMrKbPawX1XZrZwvS5bDKzO/yuJ1vMbKqZPWNm69L/Hm9P7x9rZk+Y2cb045hhf5hzrmh+gN8B16S3/wL4fZ/t3+LNGnkBsNzvWod5nm8BngRK0s/Hpx/nA6uBEmAmsBkI+l3vMM91Kt60FNuA2kL7PoG3AaH09teArxXid4k3rchmYBYQSZ/bfL/rytK5TQLOSW9XAhvS39/XgTvS++/o/W6H81NULXS8KX2r0tvVvDlr5CLgXudZBtSY2SQ/CsySW4GvOudiAM65fen9i4AHnXMx59xWYBPeAib57FvA5zhyuuaC+T6dc79z3hoDAMvwZjuFwvsuM1lIJy8553Y7515Kb7cB6/DWkOi7MNBPgHcP97OKLdA/DXzDzHYA/wrcmd6fySIe+WQucKmZLTezP5jZeen9BXWeZnYtsNM5t7rfSwV1nn18HO83Dyi8cyy08xmQmc0AzgaWAxOcc7vBC31g/HDfP6PJufKJmT0JTBzgpbuAq4D/7Zx72Mw+gDdL5NVkuEBHLjnBeYaAMXjdDecBD5nZLArvPP8er0viqD82wL6cPc/jnaNz7tfpY+4CEsDPev/YAMfn7DlmoNDO5yhmVgE8DHzaOXdoJFbpLLhAd85dfazXzOxe4Pb0018AP0hvZ7KIR045wXneCvzKeZ1zK8wshTcZUMGcp5mdjtd3vDr9D2MK8FL6QndenefxvksAM7sReCdwVfo7hTw7xwwU2vkcwczCeGH+M+fcr9K795rZJOfc7nSX4L5jv0Nmiq3LZRdweXr7SmBjensp8JH06IgLgNbeX4Xy1KN454eZzcW7yLQf7zwXm1mJmc0E5gArfKtyGJxzrzrnxjvnZjjnZuAFwjnOuT0U0PdpZguBzwPXOuc6+7xUMN9lWiYL6eQl81ocPwTWOee+2eelvgsD3Qj8erifVXAt9BP4BPDt9DJ53cDN6f2P4Y2M2AR0Ah/zp7ysuQe4x8zWAD3AjemW3Vozewh4De/X979xziV9rHOkFNL3+R94I1meSP8mssw5d4vzFpkpmO/SHWMhHZ/LypaLgRuAV83s5fS+vwe+itcdehOwnSysKaFb/0VECkSxdbmIiBQsBbqISIFQoIuIFAgFuohIgVCgi4gUCAW6iEiBUKCLiBSI/w95rK8q0XWAzgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.metrics import precision_score\n",
    "from sklearn.metrics import recall_score\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "precision_scores = []\n",
    "recall_scores = []\n",
    "\n",
    "thresholds = np.arange(np.min(decision_scores), np.max(decision_scores), step=0.1)\n",
    "for threshold in thresholds:\n",
    "    y_predict = np.array(decision_scores >= threshold, dtype='int')\n",
    "    precision_scores.append(precision_score(y_test, y_predict))\n",
    "    recall_scores.append(recall_score(y_test, y_predict))\n",
    "\n",
    "plt.plot(thresholds, precision_scores)\n",
    "plt.plot(thresholds, recall_scores)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### precision-recall曲线"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAUQUlEQVR4nO3df3Cd1X3n8fdX8i8Mxja2AeMfiATTwbhtIFoWwm5IGpIAO2unnaSDm2ybLBM63ZDNNJnMspMOzdL0jw2T6TQd0sa7TVMymwDNtqmbmnE3LGyyCWYtAiFg6la4NpblYGFZ/iVLsqzv/nFFkWVJ98q60r330fs1o5n73Ofc5345yB8fn3vucyIzkSQ1vqZaFyBJqg4DXZIKwkCXpIIw0CWpIAx0SSqIObV64+XLl2dLS0ut3l6SGtKzzz77emauGOtczQK9paWFtra2Wr29JDWkiNg33jmnXCSpIAx0SSoIA12SCsJAl6SCMNAlqSDKBnpEfC0iDkXEi+Ocj4j4ckS0R8QLEXFD9cuUJJVTyQj968DtE5y/A1g3/HMP8MdTL0uSNFll16Fn5vcjomWCJpuAh7N0H94dEbEkIlZm5sEq1XiWnXu7+cE/dE3HpQtjw6rFvO+6y2tdhqQZVo0vFq0C9o847hh+7pxAj4h7KI3iWbt27Xm92Y/3HeGPnmw/r9fOBplwxeIFBro0C1Uj0GOM58bcNSMztwBbAFpbW89rZ43fvPWt/Oatbz2fl84Kv/3o8zy770ity5BUA9VY5dIBrBlxvBrorMJ1dR76Tp9h/hwXL0mzUTX+5G8Ffn14tctNwNHpmj9Xef2DQ8yf28TgmaFxf9x2UCqmslMuEfEt4F3A8ojoAH4XmAuQmX8CbAPuBNqBXuBj01WsyhscSl48cIyrP/f4uG3+zS+s5KFfc3WpVDSVrHLZXOZ8Ap+oWkWakk+/9xr+xZVLxz3/necP8MqhEzNYkaSZUrPb52p6vG3NEt62Zsm453964CivdvfOYEWSZoqfns0yEaWljZKKxxH6LBMEXSf6eXD739e6lEKZ19zMh29ay/KL5te6FM1iBvosc+3Ki/ney6/x1f+zp9alFMaZTDLh2pWL/EKXaspAn2U+dds6PnXbulqXUSgff7iNZ/cd4Z3XjLnNozRjnEOXpmB/dy/fe/k1fu3GtSyY21zrcjTLOUKXpuDhp/cSwIdaV9M/eKZq153X3ETEWHfVkMZnoEtT8Dc/OchQwq0PPlXV6/7bX7yCP9p8fVWvqeIz0KUp+C+brqO9yl/U2rm3m8d/epDujddxyYXzqnptFZuBLk3B+6+7nPdfV91r7uo8xlO7f8C2nx7kIzddWd2Lq9D8UFSqM9euXMS6Sy9i6/PetFST4whdqjMRwcZfvIIv/a9/4Avf3UVz87kfjjZHsPnGtay5ZGENKlS9MtClOvTLN6ziz5/exzd27Dvn3JmhZHAoWbnkAv6dUzIawUCX6tDqpQtp+53bxjz3+a0v8T+e2ccdG/xWqs7mHLrUQHoHBvmfP+7gjg0rvW+MzuEIXWogW5/v5HjfIL9ywyp6Bwarcs05TU3Mc9vCQjDQpQby+Is/A+Cjf7azatdctGAO//c//RKLL5hbtWuqNgx0qYF86rZ1vOOty6p2vYef3kf/4BkWzvM+NEVgoEsN5Ia1S7lh7fhbDE7G4RP9PLh9N//+X13F3GanXIrA/4vSLPVXzx1gcCj50NtX17oUVYkjdGmW+vazHSyc18yjO/fXupSyNqxazAeuX1XrMuqegS7NUhfMayaAb/2/V2tdyoR6T59h7SULDfQKGOjSLPVX/+GWWpdQ1tBQ0vr73+PtVfrcoOicQ5dUt/7+Z8fpPjnAO65eXutSGoKBLqlu/eiV1wG45erqLdUsMgNdUt360SuHecvyC1m5+IJal9IQDHRJden0mSGe2XOYm6v4RaqiM9Al1aUXOno4OXCGW5w/r5iBLqku/aj9MBFw81scoVfKZYuS6tIPX3mdhXOb+cMn/nHG3vPKZQv52C1Xzdj7VZuBLqkuXbpoAc1Nx/jLH3fMyPv1nR5iKJPfuLmFpqZzt/1rBAa6pLr05c3Xz+j7feKbP+bFA0cbNsyhwjn0iLg9InZHRHtE3DfG+bUR8WREPBcRL0TEndUvVZKmz97XT9Ky7MJalzElZUfoEdEMPAS8F+gAdkbE1szcNaLZ7wCPZeYfR8R6YBvQMg31SlLVZSb7Dvey4YrFHDk5MGabCFh8wVwi6ncEX8mUy41Ae2buAYiIR4BNwMhAT+Di4ceLgc5qFilJ0+nkwBlO9A/yaNt+Hm0b/+6Tn33/z/GJd189g5VNTiWBvgoY+V/YAfzLUW0+D/xdRHwSuBAYc7vyiLgHuAdg7dq1k61VkqbFRfPn8JUP38ChY31jnh8cSr7wty9zauDMDFc2OZUE+lj/vshRx5uBr2fmlyLiZuAbEbEhM4fOelHmFmALQGtr6+hrSFLN3PnzK8c919lzii/87ctcsaS+b0FQyYeiHcCaEcerOXdK5W7gMYDMfBpYAPj1LkmFcPDoKQCuWLKgxpVMrJJA3wmsi4irImIecBewdVSbV4H3AETEtZQCvauahUpSrRzoKU3FrGr0EXpmDgL3AtuBlymtZnkpIh6IiI3DzT4DfDwifgJ8C/hoZjqlIqkQOntKI/SVdR7oFX2xKDO3UVqKOPK5+0c83gXU//YnknQeOntOsfiCuVw0v76/i+nNuSSpjM6eU6xcXN/z52CgS1JZB3r66n7+HAx0SSrr4NFTdb9kEQx0SZrQyf5BenpPG+iS1OgaZQ06GOiSNKE31qA7QpekBvfGGnQDXZIaXGfPKZoCLls0v9allGWgS9IEOnv6uPziBcxprv+4rP8KJamGOnsaY8kiGOiSNKHOo6fq/h4ubzDQJWkcQ0PJwZ6+hliyCAa6JI3rteN9DJwZYs3ShbUupSIGuiSN49XDvQBcucxAl6SGtq+7FOhrL2mMQK/vm/tKUg3tHw70P3ziH5nbNPH49x1XL2PT21bNRFnjMtAlaRxvXXERKxcv4Efthyds131ygBc7jxroklSvPnD9Kj5wffmQ/uWv/JAL59U+Tp1Dl6Qp6j45wCUXzqt1GQa6JE1V9wkDXZIaXv/gGY73DxroktToenpPAxjoktToDp8YAGCZgS5Jja37ZCnQHaFLUoPr7jXQJakQuk/0Awa6JDW87pMDRMCShQa6JDW07t4Bli6cR3NT1LoUA12SpqJeviUKBrokTcnhEwNcUgfTLWCgS9KUdB3vZ8XF82tdBlBhoEfE7RGxOyLaI+K+cdr8akTsioiXIuKb1S1TkurTa8f6uGxRfew5WvZ+jxHRDDwEvBfoAHZGxNbM3DWizTrgPwO3ZOaRiLh0ugqWpHpxon+QkwNnuLSBRug3Au2ZuSczB4BHgE2j2nwceCgzjwBk5qHqlilJ9efQsT4ALmugQF8F7B9x3DH83EjXANdExA8jYkdE3D7WhSLinohoi4i2rq6u86tYkurEoeOlLxVdWidTLpUE+liLK3PU8RxgHfAuYDPw3yNiyTkvytySma2Z2bpixYrJ1ipJdeW1BhyhdwBrRhyvBjrHaPPXmXk6M/8J2E0p4CWpsLreGKFf3Dgj9J3Auoi4KiLmAXcBW0e1+Q7wboCIWE5pCmZPNQuVpHrz2rE+FsxtYtH82u8nChUEemYOAvcC24GXgccy86WIeCAiNg432w4cjohdwJPAZzNz4m2yJanBHTrez2UXLyCi9l/7hwqWLQJk5jZg26jn7h/xOIFPD/9I0qzw2rE+Ll1UH/Pn4DdFJem8HTrWXzfz52CgS9J5O3S83xG6JDW6k/2DnOgf5DJH6JLU2A4eLa1Bv9xAl6TGdqDnFACrll5Q40reZKBL0nk4cGQ40JcY6JLU0A709DKnKZxDl6RGd+DIKS5fvKAu9hJ9g4EuSeehs6ePK+pougUMdEk6Lwd6TrHaQJekxjZ4ZoifHeurqxUuYKBL0qT97FgfZ4ayrla4gIEuSZP2ancvAKuXLqxxJWcz0CVpkvYdLgV6y3IDXZIa2t7XTzJvThNXLHbKRZIa2t7DJ1l7yUKa6mgNOhjokjRpe1/vpWXZhbUu4xwGuiRNwtBQsq/7JC3L6mv+HAx0SZqU14730Xd6iJbljtAlqaHtfX14hUsdTrlUtEm0JKlk7+GTAHzlqXYefnrvpF8/d04Tn33fz03LCN9Al6RJuOayRWxYdTHdJwfoPjkwqdcODiXth05w81uWGeiSVGtvv3Ip3/3kvz6v1x463seNv/9ElSt6k3PoklQQBrokFYSBLkkFYaBLUkEY6JJUEAa6JBWEgS5JBWGgS1JBVBToEXF7ROyOiPaIuG+Cdh+MiIyI1uqVKEmqRNlAj4hm4CHgDmA9sDki1o/RbhHwH4Fnql2kJKm8SkboNwLtmbknMweAR4BNY7T7PeCLQF8V65MkVaiSQF8F7B9x3DH83D+LiOuBNZn53SrWJkmahEoCfaxN8/KfT0Y0AX8AfKbshSLuiYi2iGjr6uqqvEpJUlmVBHoHsGbE8Wqgc8TxImAD8FRE7AVuAraO9cFoZm7JzNbMbF2xYsX5Vy1JOkclgb4TWBcRV0XEPOAuYOsbJzPzaGYuz8yWzGwBdgAbM7NtWiqWJI2pbKBn5iBwL7AdeBl4LDNfiogHImLjdBcoSapMRRtcZOY2YNuo5+4fp+27pl6WJGmy/KaoJBWEgS5JBWGgS1JBGOiSVBAGuiQVhIEuSQVhoEtSQRjoklQQBrokFYSBLkkFYaBLUkEY6JJUEAa6JBWEgS5JBWGgS1JBGOiSVBAGuiQVhIEuSQVhoEtSQRjoklQQBrokFYSBLkkFYaBLUkEY6JJUEAa6JBWEgS5JBWGgS1JBGOiSVBAGuiQVhIEuSQVhoEtSQRjoklQQFQV6RNweEbsjoj0i7hvj/KcjYldEvBART0TEldUvVZI0kbKBHhHNwEPAHcB6YHNErB/V7DmgNTN/Afg28MVqFypJmlglI/QbgfbM3JOZA8AjwKaRDTLzyczsHT7cAayubpmSpHIqCfRVwP4Rxx3Dz43nbuDxsU5ExD0R0RYRbV1dXZVXKUkqq5JAjzGeyzEbRnwEaAUeHOt8Zm7JzNbMbF2xYkXlVUqSyppTQZsOYM2I49VA5+hGEXEb8Dng1szsr055kqRKVTJC3wmsi4irImIecBewdWSDiLge+CqwMTMPVb9MSVI5ZQM9MweBe4HtwMvAY5n5UkQ8EBEbh5s9CFwE/EVEPB8RW8e5nCRpmlQy5UJmbgO2jXru/hGPb6tyXZKkSfKbopJUEAa6JBWEgS5JBWGgS1JBGOiSVBAGuiQVhIEuSQVhoEtSQRjoklQQBrokFYSBLkkFYaBLUkEY6JJUEAa6JBWEgS5JBWGgS1JBGOiSVBAGuiQVhIEuSQVhoEtSQRjoklQQBrokFYSBLkkFYaBLUkEY6JJUEAa6JBWEgS5JBWGgS1JBGOiSVBAGuiQVhIEuSTNkfnMzd/785ay9ZOG0XL+iQI+I2yNid0S0R8R9Y5yfHxGPDp9/JiJaql2oJDW6xQvn8pUPv513XrNiWq5fNtAjohl4CLgDWA9sjoj1o5rdDRzJzKuBPwD+a7ULlSRNrJIR+o1Ae2buycwB4BFg06g2m4A/H378beA9ERHVK1OSVE4lgb4K2D/iuGP4uTHbZOYgcBRYNvpCEXFPRLRFRFtXV9f5VSxJGlMlgT7WSDvPow2ZuSUzWzOzdcWK6ZlDkqTZqpJA7wDWjDheDXSO1yYi5gCLge5qFChJqkwlgb4TWBcRV0XEPOAuYOuoNluB3xh+/EHgf2fmOSN0SdL0mVOuQWYORsS9wHagGfhaZr4UEQ8AbZm5FfhT4BsR0U5pZH7XdBYtSTpX2UAHyMxtwLZRz90/4nEf8KHqliZJmoyo1cxIRHQB+2ry5tWzHHi91kXUEfvjTfbF2eyPs02lP67MzDFXldQs0IsgItoys7XWddQL++NN9sXZ7I+zTVd/eC8XSSoIA12SCsJAn5ottS6gztgfb7IvzmZ/nG1a+sM5dEkqCEfoklQQBrokFYSBXoEKNvj4dETsiogXIuKJiLiyFnXOhHJ9MaLdByMiI6LQS9Uq6Y+I+NXh34+XIuKbM13jTKrgz8raiHgyIp4b/vNyZy3qnAkR8bWIOBQRL45zPiLiy8N99UJE3DDlN81Mfyb4oXS7g1eAtwDzgJ8A60e1eTewcPjxbwGP1rruWvXFcLtFwPeBHUBrreuu8e/GOuA5YOnw8aW1rrvG/bEF+K3hx+uBvbWuexr7453ADcCL45y/E3ic0t1qbwKemep7OkIvr+wGH5n5ZGb2Dh/uoHRHyiKqZLMTgN8Dvgj0zWRxNVBJf3wceCgzjwBk5qEZrnEmVdIfCVw8/Hgx5965tTAy8/tMfNfZTcDDWbIDWBIRK6fyngZ6eZVs8DHS3ZT+1i2isn0REdcDazLzuzNZWI1U8rtxDXBNRPwwInZExO0zVt3Mq6Q/Pg98JCI6KN0f6pMzU1pdmmy2lFXRzblmuYo27wCIiI8ArcCt01pR7UzYFxHRRGlP2Y/OVEE1VsnvxhxK0y7vovQvtx9ExIbM7Jnm2mqhkv7YDHw9M78UETdTukvrhswcmv7y6k7F2VIpR+jlVbLBBxFxG/A5YGNm9s9QbTOtXF8sAjYAT0XEXkrzglsL/MFopZu//HVmns7MfwJ2Uwr4IqqkP+4GHgPIzKeBBZRuVDUbVZQtk2Ggl1d2g4/haYavUgrzIs+RTtgXmXk0M5dnZktmtlD6PGFjZrbVptxpV8nmL9+h9KE5EbGc0hTMnhmtcuZU0h+vAu8BiIhrKQX6bN1geCvw68OrXW4Cjmbmwalc0CmXMrKyDT4eBC4C/iIiAF7NzI01K3qaVNgXs0aF/bEdeF9E7ALOAJ/NzMO1q3r6VNgfnwH+W0T8NqXphY/m8JKPoomIb1Gaals+/JnB7wJzATLzTyh9hnAn0A70Ah+b8nsWtC8ladZxykWSCsJAl6SCMNAlqSAMdEkqCANdkgrCQJekgjDQJakg/j8H8I9yvPWqsQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(precision_scores, recall_scores)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### scikit-learn中的precision-recall曲线"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import precision_recall_curve\n",
    "\n",
    "precisions, recalls, thresholds = precision_recall_curve(y_test, decision_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(145,)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precisions.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(145,)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recalls.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(144,)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "thresholds.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "the last precision and recall values are 1. and 0. respectively and do not have a corresponding threshold."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3deXhU5f3+8fcn6yRkgZAFzMIaBERkGRYrKiIiohaqWHFfUESrba1dbF1at2+r1vandUUFRW2tWiuoFBBcQGULyJZgIOwhkIUtG9mf3x8naAwDmYSZnFk+r+uaazIzJzM343jzcOac5xFjDEoppfxfiN0BlFJKeYYWulJKBQgtdKWUChBa6EopFSC00JVSKkCE2fXCiYmJpnv37na9vFJK+aXVq1eXGGOSXD1mW6F3796drKwsu15eKaX8kojsPN5justFKaUChBa6UkoFCC10pZQKEFroSikVILTQlVIqQLRY6CIyU0SKRGTjcR4XEXlGRPJEZL2IDPF8TKWUUi1xZ4T+GjD+BI9fBGQ2XqYBL5x8LKWUUq3V4nHoxpglItL9BJtMBGYbax7e5SLSUUS6GmP2eijjD+1cBls/hXN+A2ERXnkJpXzRqh0HWLq52O4YygPO75fCGekdPf68njixKBXY3eR2fuN9xxS6iEzDGsWTkZHRtlfbvQKWPAGjfglooavg8eSCXFZuP4CI3UnUyUqOc/hsobv6eLlcNcMYMwOYAeB0Otu2ssbRT7MuzKGCTGVNHWP6JjPzxmF2R1E+yhNHueQD6U1upwEFHnje4zj694cWugouVbUNOML1wDR1fJ74dMwFrm882mUkcNhr+8/h+xF6fS3U17m+6OhdBaCq2nocYaF2x1A+rMVdLiLyL2A0kCgi+cAfgXAAY8yLwDxgApAHVAI3eSssACHh1vUTPY6/Te8L4Nr3vBpDqfZWVdtAZLgWujo+d45yuaqFxw3wM48lasmAy6HuiDUSd2X3cti6GKoOgyO+3WKp4GaMYWtxBSIQGRaCIzzUuoSFEBbqmd0kVbX1RGmhqxOwbfrcNotJglF3H//x7Usgb5F1eOOpJzp8XinPqK1v4NfvrmPOWtdfHSXGRHD50DQSO0QS4wgj1hFGeqfoVh/lUFVbr/vQ1Qn5X6G3JG0YhEbAjqVa6Mrrqmrr+dlba1j8bRG3j+5F3y6xVNc2UFVXT1VtPUdqGliQvY+Xvth2zO8mdIigW+douiVEk9G5A13iHN8VfmxkGLGOcGIdYSTFRiJAXYPBoSN0dQKBV+jhUZA2HHLmQlik3Wns0+9SOGWw3SkCWnl1Hbe8vooV2w/w6KQBXDuym8vtfjE2k4YGQ0VNHeXVdZQeqWPZ1hJyC8vYub+SVTsOMmddwXG/y+8QEcrInp0BdISuTijwCh3g9Mvhf7+Dr562O4k9GuphzWy4aw044uxOE5AOVtRw46yVbCwo5f9dOYiJg1JPuH1IiDSOuMPpGg+ndon9wePVdfUcrKilrKqWsuo6yqrqKK+qo7Sqlg17DvNFrnWGaJf4KK/9mZT/E2PTIX5Op9PoEnResmc1vDwGzv41nP+A3Wn8wv7yahZvKmJhzj7CQkJ44oqBxDnCXW5bWFrFta+sYOeBSp6/eghj+6d4PZ8xhuKyahJjIgkJ0VNFg5mIrDbGOF09Fpgj9GCXOhROvwKWPQvOmyH+xKPHYLVrfyULc/axMLuQrJ0HaGgytvk0t4junaPJSOjAKR0dJMZEkhgTSVxUGE/Mz2V/eTWv3TSMH/VKbJesIkJynKNdXkv5Ly30QDXmAet7hMUPw4+fad3vSgiEuh6d+jNjDNkFpSzM3sfCnEK+3VcGQN8usdw1JpNxp6XQv2sc2QWlzF1XwPaSCnbtr2Tl9v2UVn1/mGzH6HDeunUkg7wwF4dSJ0MLPVB16gYjp1vfI6x/u3W/KyFw9buQOdY72dpRXX0DK3ccYGF2IZ/kFLLn0BFCBJzdE7j/4n6M69+FjM7RP/idAanxDEj94TkMNXUN7K+opqSshtROUSR00InhlO/RQg9ko38P8elQXer+79TVwBd/gZLNflPo1XX1fLmlhM4xkaR1iiI6IpQlm0tYmLOPT78t4lBlLZFhIZydmcQvxmZyft9kOse07gioiLAQusZH0VW/lFQ+TAs9kIVHwfBbW/c79bVWoddUeCeTF7yblc/9Hxy7oFZ8VDjn90tmXP8unNMnkegI/birwKafcPVDoeHWfDm1/lPoy7btp0ucg0cnDWD3wUoOVtYyskcCw3okEO6h0+6V8gda6OpYER1g62dgGqxdNs6pEOKbxWiMYeX2A/yoV+d2OXxQKV+mha6OlT7cmhOnaBPUV1vTEY+YZncql3bsr6S4rJrhPRLsjqKU7bTQ1bGuede6NgbevBwW/REyL4CEE0xZbJMV2/YDMEILXSmPLHChApWIdQx7SBjMvQsaGuxOdIyPN+wltWMUvZJi7I6ilO200NWJxafBhY9Zs1eunGEd/dL8UnvElmgFh47wZV4Jk4emIbpyslK6y0W5YfB1kDMH5v/Ourgy6QUYdHW7xnp/TT7GwOVD0tr1dZXyVVroqmUicPkrsO7f1pekzX3zFix9CgZOabejYYwxvLc6nxE9Eo4501OpYKWFrtwT1cmaSsCVuFT4z1TYsrDdFhXZfeAIO/ZXMvXsnu3yekr5A92Hrk5e/4lWqS97tt1ecu9ha799dx2dK/UdHaGrkxcaDiOmwycPwEe/sqYcaKvwKDjrFxAZe8LN9pVWAdBFp5RV6jta6Mozht4Aa16H9f9u+3M01EPdEThlCPSdcMJN9x22Cj0lXgtdqaO00JVnOOLhrtUn9xzFufDccKitPO4mtfUNzF62k2c/yyMlLpLYSP0IK3WU/t+gfMfRXTXHOa7989wiHvkoh63FFZydmcgfL+2vx58r1YQWuvIZB6pDSABKy8pourT11uJyHv0oh89yi+neOZpXb3Aypm+ylrlSzWihK5/x2KJdPAXM/WQRC/KGMiA1niM19by5fCdR4aH8YUJfbvxRDyLC9OAspVzRQlc+YWtxOR9kH+LMiLFcG7aI3QecvLJtEHUNhiud6dwz7lSSYlu3ypBSwUYLXfmEx//3LVHhoYy++zV473J+v+8f3HPHfMo79tP1O5Vyk/7bVdlu3oa9LMwpZPq5PUmMj4WfvgGOjkS8cy0JlNkdTym/oSN0Zatvdh3k7n+vZUhGR245ehp/bApMeRNmXgRvXgYZI1t+opgU+NFd1klOSgUpLXRlm90HKrl1dhYpcQ5evt6JIzz0+wdTh8Kk52H+72Hdv078RAaoPgx1VXDeH7yaWSlfpoWubFFZU8fNr62ipq6Bt6cNo3OMiy88T59sXdzx3+mw5K+QeSGkDfVsWKX8hFv70EVkvIjkikieiNzr4vEMEflMRL4RkfUicuLztlXQ+/TbIrYUlfPUTwfRO9kDqw2N/wvEdoX/3gY1xz/TVKlA1uIIXURCgeeAC4B8YJWIzDXG5DTZ7H7gHWPMCyLSH5gHdPdCXhUglmwuJtYRxnmnJnnmCaM6wqTnYPZE+OTBk9v1Ehmr++KVX3Jnl8twIM8Ysw1ARN4GJgJNC93Adyf3xQMFngypAosxhiWbSxjVO5GwUA8eaNVztDXr44oXYdXLbX+ermfAbUs8lUqpduNOoacCu5vczgdGNNvmT8BCEbkL6ACMdfVEIjINmAaQkZHR2qwqAFTV1vPQh9nsK61iTN9kz7/ABQ9bhVzdxsMdc+ZAYbZnMynVTtwpdFcTZphmt68CXjPGPCUiZwJviMgAY8wPlok3xswAZgA4nc7mz6EC3I6SCu54aw05e0u5fXQvLvPGWqBhkSe3tunBnbB3nefyKNWO3Cn0fCC9ye00jt2lMhUYD2CMWSYiDiARKPJESOX//rdhL799bz0hIcLMG52M6ZtidyTXQsOgvsbuFEq1iTs7MFcBmSLSQ0QigCnA3Gbb7ALOBxCRfoADKPZkUOW/lm4p5va31tAzOYaPfz7Kd8scICQc6mvtTqFUm7Q4QjfG1InIncACIBSYaYzJFpGHgSxjzFzgHuBlEbkba3fMjcYY3aWiAPgqbz/hocK/p4384clDviiiA2CgpqLxZ6X8h1snFhlj5mEditj0vgeb/JwDnOXZaCpQ5OwtpU9KrO+XOVjHsgOU7YPOvezNolQr6eRcyquMMeQUHKZf17iWN/YFsV2s67K99uZQqg200JVXbS0up6S8hqHdOtkdxT1NR+hK+RktdOVVX2/dD8CPenW2OYmb4hoL/XC+vTmUagMtdOU13+4r5Y1lO0ntGEVGQrTdcdzjiIcOyVCyxe4kSrWazraoPK6ypo6nF23hlS+3E+cI48nJZ/jXgs7J/aAop+XtlPIxWujKoxZvKuTBOdnsOXSEK53p3HtRXzr52xJyyf1hzWxoaIAQ/Ues8h9a6MpjPs8tYurrWWQmx/DObWcyvEeC3ZHaJrkf1FbAoZ2Q0MPuNEq5TYcf6qR9nVdCcVk1C3MKiYkM46Ofj/LfMgfIOBMQyJppdxKlWkVH6OqkvLF8Jw98sJGo8FDCQoWRPROIDPODE4hOJKkPDLrGmobXebOO0pXf0BG6arOlW4r509xszumTxIWnpVBeXccF/X14npbWGHO/Na/LJw+2vK1SPkJH6KpN8orKueOtNWQmx/D8NUOIiQzj4UkDiI0MkI9UXFcY9Uv47DHY8RV015ktlO/TEbpqtQMVNUx9fRWRYSG8coOTmMYSj3OE+9fhiS05806IS4UPfwGVB+xOo1SLtNBVq1TX1TP9jdXsPVzFjOudpHXykxOG2iIiGi6bYR3t8q+roPaI3YmUOiEtdNUqH3yzh5U7DvDk5IEMyfCT+VlORvdR8JOXYPcK+M8t0FBvdyKljksLXbXKtpIKIkJDuHTgKXZHaT8DLoPxf4ZvP4L//RZ0qn/lowLkGyzVXvYeqqJLvIOQkADaV+6OkbdDaQF8/Yx1He1isrGoTjDqboj242PwlV/TQletsnHPYTKTY+yOYY+xD0FdNWz60PXjFUWw8X244jVIH9au0ZQCLXTVCkWlVWwrqWDK8PSWNw5EISEw4Qnr4sqe1fDuTTBrPIz9k3WUTCAd9aN8nu5DV8dV32DYUVLB0eVh56wtAOCs3ol2xvJdqUPhtiXQZzwsvN86MkYPd1TtSEfo6rhmfbWdRz/eRK+kDlw2JI1ZX21nVO9ETjsl3u5oviuqI1z5Jqx4ySr1l86BybN0F4xqFzpCVy4ZY3g3K58eiR3o3CGSJxfkUlJewy/HZtodzfeJwMjpMHWB9fPsH0NVqd2pVBDQEbpyaV3+YXILy3hk0gCuG9mNXfsryT9YibO7HsHhttShMOYBeP9WKC8Eh58slK38lha6cunZT7cQHxXOpEHW8eYZnaPJ6BzAZ4V6S2RjiesIXbUD3eWijrFpbymLNhVx69k9iHWE2x3Hvzkav2+oPmxvDhUUtNDVD6zdfYiLnl4KwDUjutmcJgA4dISu2o8WuvrO6p0Hufrl5d/d9ru1QH3R0V0u1Vroyvt0H7oCoLSqlhtnriQlzsGlA7syKjPJ7kiB4bsRuu5yUd6nha4A2LW/krLqOp68YiDjB3S1O07giIgFRHe5qHahhR7kisuq+Xh9Ae+tyQcgKdZhc6IAExICkbG6y0W1Cy30IFRVW89H6/cyZ+0evsorocFAv65x3DehH2ek6VmgHhfd2VrGrrrMKnelvEQLPQj9/ZPNvLRkG+kJUdwxujc/HnQKfVK0aLxm3KPwzvXw5mS49j0tdeU1bh3lIiLjRSRXRPJE5N7jbPNTEckRkWwR+adnYypPKimvoWu8gyW/OY9fX3iqlrm39bsEJs+E/FXw1k+hutzuRCpAtVjoIhIKPAdcBPQHrhKR/s22yQR+D5xljDkN+KUXsioP2VpcTmJMZGAt6OzrTpsEl79iLWX3z59CTYXdiVQAcmeEPhzIM8ZsM8bUAG8DE5ttcyvwnDHmIIAxpsizMZUnVFTX8c8Vu1i7+xATBwXREnK+YsBl1qLTu5ZZU+vqUnbKw9zZh54K7G5yOx8Y0WybPgAi8hUQCvzJGDO/+ROJyDRgGkBGRkZb8qpWqm8wfJVXwn+/2cP8jfs4UltPZnIMk4em2R0tOJ0+GQ5sh88etZayi0+1O5EKIO4Uuqt/lzcfWoQBmcBoIA1YKiIDjDGHfvBLxswAZgA4nU4dnnjJut2HqK5rYHBGR3787Fds2ltKrCOMSYNTuWxIKs5unXR3i52S+ljXRw5ooSuPcqfQ84Gma46lAQUutllujKkFtotILlbBr/JISuW23Qcqufrl5VTU1NM7OYa8onJ+PqY3d5zXG0d4qN3xFEBU4xTEupqR8jB39qGvAjJFpIeIRABTgLnNtvkAOA9ARBKxdsFs82RQ1bKGBsNv3luHiHDfhH7U1jcAcOkZp2iZ+5KoTtb1kYP25lABp8URujGmTkTuBBZg7R+faYzJFpGHgSxjzNzGx8aJSA5QD/zGGLPfm8HVsWYv28HybQd4/PLTuXJYBjeP6kHBoSOkJ+g85j4lJtm6LtpkHf2ilIeIsembdqfTabKysmx57UB0pKaeYY8tYmi3Trx20zDdR+7r/jkFdnwJP1/zfcEr5QYRWW2Mcbp6TKfPDRAfriugvLqO6ef20jL3B+Mehboj8NljdidRAUQLPQB8lVfC/XM2MjAtnuE9dM1Pv5DYG4ZPgzWzYd9Gu9OoAKGF7ueMMdz977V07xzN6zcNJzRER+d+49zfWkvULfiD3UlUgNBC93PLtx2gqKya68/srisM+ZuoTnD2PbD9C9i73u40KgBoofspYwyzvtrO9TNXkNYpinH9U+yOpNpi8LUQ5oDVs+xOogKAFrofMsbw1MLNPPRhDuf2Sebju84mOU4XpvBLUZ1gwOWw/h1rvnSlToIWuh96a8Uunv0sj6uGpzPjuqHER4fbHUmdDOfNUFNulbpSJ0EL3c9sLS7n759sZmTPBB6bdDoh+iWo/0sdCl0GWocwbvvc7jTKj2mh+wljDM9/nsdFTy+ltr6BBy7pr2UeKETg8lchOhFmT4LP/wIN9XanUn5IC91PZBeU8sT8XM7JTGLRPedy2im69mdASeoDt34KA6+Ez/8Mb14G5bqsgGodLXQfd6Smnv+szueBOdbJJ9PP7UlyrH4BGpAiY+AnL8Klz8Cu5fDi2db0AEq5SQvdh336bSHDHlvEPe+u42BFDfde1Jcz0jvaHUt5kwgMvQFuWQQRHeD1S2H163anUn7CnfnQlQ3Kq+t45KNNpMRF8upPnAzvkaBztASTLqfDtM/hrSvgs/+DwddBiI6/1InpJ8THNDQYVu88yBUvLmPXgUoeuKQ/I3p21jIPRo4465DG8n2wZ7XdaZQf0BG6j8jacYC56wpYkL2PwtJqYiPDePUGJ6NP1alVg1qfcRASBt9+COnD7E6jfJwWug94Y9kOHpiTjSM8hHP7JDF+QBfG9E0hPkpPGAp6UZ2gxzmw6SMY+5C1j12p49BCt1FVbT1PzM9l5lfbGdsvmWeuGkx0hP4nUc30vQQ+/hUU50JyX7vTKB+m7WGT4rJqbpy1kuyCUq4/sxt/mNBP1/1UrmWOs67zFmmhqxPSL0Vt0NBguO7VFWwtLueV6508PHGAlrk6vo7pkHiqVehKnYAWug3eXb2bb/eV8X8/OZ2xOu2tckfvsbDza6iptDuJ8mFa6O1s7roCfvefDQzvnsDFA7vaHUf5i97nQ301vHcTHN5jdxrlo7TQ21F9g+GlL7bSJyWG2VOHExmmu1mUm7qdBT3Pgy0L4emB8N5UPTZdHUMLvZ0YY3hiwbdkF5Ry2zm9dJ+5ap1wB1z/Afx8LYyYbhX7y2Ng5njImauzMypAC73dvLx0Gy99sY1rRmRw2ZBUu+Mof9WpG1z4GNydDRf+GUr3wDvXwTODYfkLuupRkNNCbwdrdx/i8fm5TDi9C49MHKCn8auT54iDM++wRuw/nQ2xXWH+vfC3/rDgPt3PHqS00L2stKqWe/+znoQOETx++UBdlEJ5Vkgo9J8IUxfALZ9C5gXWSP3VC6Cuxu50qp1poXvZox/lsLW4nCcmDyTWoafyKy9KGwqTZ8KUf1q7YnLm2J1ItTMtdC9avm0/C3MKOat3IufpJFuqvWSOg4ResHKG3UlUO9NC95J5G/Zy7SsrSIiO4A8T+tkdRwWTkBAYfivkr4SCb+xOo9qRFrqHFZVVMXvZDn7x9jcMSu/IB3eeRZ+UWLtjqWAz6GoI7wArX7E7iWpHOjmXh3yz6yCPfbyJ1bsOYgwMTIvn1RuHEaf7zZUdHPHQ50LY/oXdSVQ7cqvQRWQ88DQQCrxijPnLcbabDLwLDDPGZHkspQ86VFnDyu0HWL3rIGt2HmT1zoN0iXNw99g+XNA/hb5dYvXwRGWvLgMg+32oOmwVvAp4LRa6iIQCzwEXAPnAKhGZa4zJabZdLPBzYIU3gvoKYwzvrs7nobnZVNTUEx4qnHZKPLed24vbR/fSEbnyHcmnWddFmyBjpL1ZVLtwZ4Q+HMgzxmwDEJG3gYlATrPtHgGeAH7t0YQ+orSqlmVb9/Pe6nw+ySlkZM8E7hl3Kqenxutp/Mo3pfS3rguztdCDhDuFngrsbnI7HxjRdAMRGQykG2M+EpGAK/Tfv7+ed7LyqW8wREeEcu9Ffbn17J6E6klCypfFp0NUAqx82TqUsWO63YmUl7lT6K5ay3z3oEgI8HfgxhafSGQaMA0gIyPDvYQ22lZczlOfbObj9XsZ0SOBuy/ow5CMTkSE6cFByg+IWCcavXM9vHI+XPU2pA6xO5XyIneaKR9o+ld7GlDQ5HYsMAD4XER2ACOBuSLibP5ExpgZxhinMcaZlJTU9tTtYEdJBde9upIvcou59ewevH7zcEb27KxlrvxLr/Ng6icQFgmzJlgzM6qA5c4IfRWQKSI9gD3AFODqow8aYw4DiUdvi8jnwK/97SiXkvJqFm8qZMOew2zcU0p2wWEcYaHMnjqcIRmd7I6nVNsl94VbFsPbV1szM459CM76hTWCVwGlxUI3xtSJyJ3AAqzDFmcaY7JF5GEgyxjj93/lG2O45uUV5BaWERsZRv9T4rjprB7cfFYPusQ77I6n1MmLSYYbPoQPbodFf4TSApjwhN2plIe5dRy6MWYeMK/ZfQ8eZ9vRJx+rfW0uLCe3sIzbzunJ78b31RkRVWAKj4LLZ0JMCqx4EbqfZc3UqAJG0O8Qrq6r565/rSHOEca1I7tpmavAFhIC4x6FroPgw19CWaHdiZQHBXWhz9+4l0nPfc3mwnLuv7g/6QnRdkdSyvtCw+GyGVBTAR/+HIxp+XeUXwjKQj98pJa/Lcxl+ptrKCqtYvq5vbj0jFPsjqVU+0k6Fcb+CTbPh2/esDuN8pCgmZyruq6ezfvK2bDnMC98kcfuA0c4OzOR568ZogtPqOA0YjrkzoN5v4XEPno2aQAIikKft2Evv3tvPWXVdQB0ig7nleudjO2fYnMypWwUEgKTZ8Gs8fDWT+HGj6DrQLtTqZMQcIV+oKKGypo6HOGhfPDNHjbsOcyctQVkJsfw+AV9GHBKPOkJUToTolIAMUlw3Qcwczy88RO4eQEk9rY7lWqjgCr0+gbD5Be+ZltJxXf3dY13MK5/Cndf0Id+XeNsTKeUj+qYDtc3lvrsiXDzfJ33xU8FVKG/sWzHd2UeGiI8NmkAU4b7/pwxStkuMROuex9eu8QaqU//EsL1pDp/ExCFXl1Xz5vLd/HIRzkM7daJmTcOo/RIrR6GqFRrdD0DLn4K3r/VmnI3bajdiVQr+XWhF5VW8eCcbD7fXERVbQN9u8Ty2k3DiHWEEx+lR64o1WpdTreuD27XQvdDflvou/ZXcsVLX1N6pI4rnemMykxi9KlJhIcG5aH1SnlGx27W9YHt9uZQbeJ3hb5y+wH+8ekW1uw8SIOBd6efyYBUXS9RKY+IiIaYLnBwh91JVBv43XB29rIdLN1SQkVNPbNuGqZlrpSnpfSH7P/Csuehod7uNKoV/K7Qr3CmEx8Vzid3n8PInp3tjqNU4Ln0Gej2I1jwe3h5DBR8Y3ci5SYxNk3M43Q6TVaWX62BoVTwMMYapc+/FyqKrWkCzrsPImPsThb0RGS1MeaYFeHAD0foSql2IAIDLoOfrYShN8HyF+C5EfDtx3YnUyegha6UOr6ojnDJ32DqQnDENy5jdwPUVNqdTLmgha6Ualn6cLjtCzj/QciZY51NeuSQ3alUM1roSin3hIbD2ffAFa/BntXw2sVQts/uVKoJLXSlVOucNgmuecc6+WjmhXoSkg/RQldKtV6vMXDDXKg6bJX6vo12J1JooSul2irNCTf9DyQEZl0EWz+zO1HQ00JXSrVdcj+Y+gnEp8Fbk+GbN+1OFNS00JVSJ6djurUoRvdRMOdn8Omj1olJqt1poSulTp4jHq55DwZfB0uehPenQV213amCjt/NtqiU8lGh4fDjf0Cn7vDpI1C2F659H8Ii7E4WNHSErpTyHBE459cw8XnYsRQWP2R3oqCiha6U8rzB18CwW2DZs7B5gd1pgoYWulLKO8Y9Bimnw3+nQ2mB3WmCgha6Uso7wh1wxSzry9H/3KqLZbQDLXSllPckZsLFf4WdX8IXT9idJuBpoSulvGvQ1TBwCix5ArYvtTtNQHOr0EVkvIjkikieiNzr4vFfiUiOiKwXkcUi0s3zUZVSfuvipyChJ7x/K1SU2J0mYLVY6CISCjwHXAT0B64Skf7NNvsGcBpjBgLvAfpvK6XU9yJjYPIsqDwAH9wODQ12JwpI7ozQhwN5xphtxpga4G1gYtMNjDGfGWOOLmGyHEjzbEyllN/rOhAufAy2LITVM+1OE5DcKfRUYHeT2/mN9x3PVOB/rh4QkWkikiUiWcXFxe6nVEoFhmG3WGeS7vjS7iQByZ1CFxf3uZx5R0SuBZzAk64eN8bMMMY4jTHOpKQk91MqpQKDCMR00f3oXuJOoecD6U1upwHHnCUgImOB+4AfG2N0Vh6llGvRna196crj3Cn0VUCmiPQQkQhgCjC36QYiMhh4CavMi/0uKW4AAAjGSURBVDwfUykVMKIToHK/3SkCUouFboypA+4EFgCbgHeMMdki8rCI/LhxsyeBGOBdEVkrInOP83RKqWAXkwIVxVBbZXeSgOPW9LnGmHnAvGb3Pdjk57EezqWUClSnDAZTD3vXQsZIu9MEFD1TVCnVvtKHW9e7V9ibIwBpoSul2ldMsnXW6O6VdicJOFroSqn2lz4Cti+BpU9BwVo9c9RDdAk6pVT7G3kHFG6ExQ9bl+hE6HUe9Dofeo2B2BS7E/olLXSlVPvrOhCmfwllhbDtM9j6qXXZ8K71eMoAq9h7nw8ZZ0JYpL15/YQY4/KkT69zOp0mKyvLltdWSvmghgYo3AB5i61y37UcGmohLAq6j7LKvdf51hzr4uoE9uAgIquNMU6Xj2mhK6V8UnW5NefL1saC359n3R+fbu2eybwQTp0AIcH1VeCJCl13uSilfFNkDJw63roAHNxhFXveYsj+ANbMhpvmQ7czbY3pS4LrrzallP/q1B2cN8OUt+CWRdZ9pXtsjeRrtNCVUv4npvEomPJCe3P4GC10pZT/ccRDaKQWejNa6Eop/yNijdIP7W552yCiha6U8k89z4VNH0JJnt1JfIYWulLKP415AMKjYP7vwKbDr32NFrpSyj/FpsDoeyFvEeS6XMY46GihK6X81/BpkNQX5t8LtUfsTmM7LXSllP8KDYeLHodDO2HVq3ansZ0WulLKv/UcDd3Phq//EfTL2mmhK6X83zm/gfJ9sPZNu5PYSgtdKeX/epwDacPhy6ehvtbuNLbRybmUUv5PxBql//MKmHURJJ0KHbtZMzN2TLeu41IhNLArL7D/dEqp4JF5AYz6FexYClsWWbtgmpJQiDulseQzvi/6jumN5Z/m9wtpaKErpQKDCIz94/e3a6us2RgP7bIuh3dbUwUc2gU7v4INe8A0W8s0JsUq+6Yj+47dvv85MqZ9/0ytpIWulApM4Q7o3Mu6uFJfC6UF3xf94d3W4Y+HdsPetda0Ag3N9sdHJbgu+o6No35HR1tXU9JCV0oFp9Bw6NTNurjS0GDN5nh4d7NR/i4o2WIttlFb+cPfiYhtUvKNu3US+0DmOAgJ9fofSQtdKaVcCQmBuK7WJX34sY8bA5X7j92dc/TnXcuh+rC17dCb4JK/e330roWulFJtIQIdEq1L6hDX21QdhiV/ha+fgejOcP4DXo2kha6UUt7iiIcLHraKfelfIToBzvyZ115OC10ppbxJxNrdUnUIFvwBojrBoKu98lJa6Eop5W0hoXDZy1BVCnPutEbufS/2/Mt4/BmVUkodKywSrnwTeo+F2C5eeQm3Cl1ExotIrojkici9Lh6PFJF/Nz6+QkS6ezqoUkr5vcgYuOYdSB3qladvsdBFJBR4DrgI6A9cJSL9m202FThojOkN/B143NNBlVJKnZg7I/ThQJ4xZpsxpgZ4G5jYbJuJwOuNP78HnC9i4+lSSikVhNwp9FRgd5Pb+Y33udzGGFMHHAY6N38iEZkmIlkiklVcXNy2xEoppVxyp9BdjbSbL7HtzjYYY2YYY5zGGGdSUpI7+ZRSSrnJnULPB9Kb3E4DCo63jYiEAfHAAU8EVEop5R53Cn0VkCkiPUQkApgCzG22zVzghsafJwOfGmOOGaErpZTynhZPLDLG1InIncACIBSYaYzJFpGHgSxjzFzgVeANEcnDGplP8WZopZRSx3LrTFFjzDxgXrP7HmzycxVwhWejKaWUag2xa8+IiBQDO215cUgESmx67dbwh5ya0XP8Iadm9Jy25uxmjHF5VIlthW4nEckyxjjtztESf8ipGT3HH3JqRs/xRk6dy0UppQKEFrpSSgWIYC30GXYHcJM/5NSMnuMPOTWj53g8Z1DuQ1dKqUAUrCN0pZQKOFroSikVIIKq0EXkChHJFpEGEXE2ub+7iBwRkbWNlxd9LWPjY79vXEQkV0QutCtjcyLyJxHZ0+T9m2B3pqNaWpzFF4jIDhHZ0PjeZdmd5ygRmSkiRSKyscl9CSLyiYhsabzu5IMZferzKCLpIvKZiGxq/H/7F433e/y9DKpCBzYClwFLXDy21RgzqPEyvZ1zNeUyY+OiIlOA04DxwPONi4/4ir83ef/mtby597m5OIuvOK/xvfOl46dfw/qsNXUvsNgYkwksbrxtp9c4NiP41uexDrjHGNMPGAn8rPFz6PH3MqgK3RizyRiTa3eOEzlBxonA28aYamPMdiAPa/ERdXzuLM6ijsMYs4RjZ01tupjN68Ckdg3VzHEy+hRjzF5jzJrGn8uATVhrSHj8vQyqQm9BDxH5RkS+EJGz7Q7jgjsLjdjpThFZ3/hPYFv/Gd6Er79nRxlgoYisFpFpdodpQYoxZi9YRQUk25zneHzx80jjesuDgRV44b0MuEIXkUUistHF5UQjs71AhjFmMPAr4J8iEudjGd1aRMRbWsj8AtALGIT1Xj7VXrlaYOt71gpnGWOGYO0a+pmInGN3ID/nk59HEYkB/gP80hhT6o3XcGu2RX9ijBnbht+pBqobf14tIluBPoBXvqBqS0bcW2jEa9zNLCIvAx95OY67bH3P3GWMKWi8LhKR/2LtKnL1PY8vKBSRrsaYvSLSFSiyO1BzxpjCoz/7yudRRMKxyvwtY8z7jXd7/L0MuBF6W4hI0tEvGEWkJ5AJbLM31THmAlNEJFJEemBlXGlzJgAaP4xH/QTri11f4M7iLLYSkQ4iEnv0Z2AcvvP+udJ0MZsbgDk2ZnHJ1z6PIiJYa0ZsMsb8rclDnn8vjTFBc8H6j5uPNRovBBY03n85kA2sA9YAl/paxsbH7gO2ArnARXa/n01yvQFsANY3fki72p2pSbYJwObG9+0+u/O4yNez8XO3rvEz6DMZgX9h7bKobfxMTsVa/H0xsKXxOsEHM/rU5xEYhbWrbz2wtvEywRvvpZ76r5RSAUJ3uSilVIDQQldKqQChha6UUgFCC10ppQKEFrpSSgUILXSllAoQWuhKKRUg/j9Y0QuNEVWWGQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(thresholds, precisions[:-1])\n",
    "plt.plot(thresholds, recalls[:-1])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
